{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "04b75f0f",
   "metadata": {},
   "source": [
    "# 导入模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f0a816c8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-18T11:22:46.385025Z",
     "start_time": "2023-11-18T11:22:46.373461Z"
    }
   },
   "outputs": [],
   "source": [
    "import joblib\n",
    "import warnings\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from typing import List\n",
    "import statsmodels.api as sm\n",
    "import matplotlib.pyplot as plt\n",
    "from skopt import BayesSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "pd.set_option(\"display.width\", 10000)\n",
    "pd.set_option(\"display.max_rows\", None)\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "plt.rcParams['font.sans-serif'] = ['FangSong']\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "plt.style.use(\"Solarize_Light2\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "973dd029",
   "metadata": {},
   "source": [
    "# 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "78b6e6a2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-18T10:51:07.508072Z",
     "start_time": "2023-11-18T10:51:07.426356Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>age</th>\n",
       "      <th>job</th>\n",
       "      <th>marital</th>\n",
       "      <th>education</th>\n",
       "      <th>default</th>\n",
       "      <th>housing</th>\n",
       "      <th>loan</th>\n",
       "      <th>contact</th>\n",
       "      <th>month</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>duration</th>\n",
       "      <th>campaign</th>\n",
       "      <th>pdays</th>\n",
       "      <th>previous</th>\n",
       "      <th>poutcome</th>\n",
       "      <th>emp_var_rate</th>\n",
       "      <th>cons_price_index</th>\n",
       "      <th>cons_conf_index</th>\n",
       "      <th>lending_rate3m</th>\n",
       "      <th>nr_employed</th>\n",
       "      <th>subscribe</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>51</td>\n",
       "      <td>admin.</td>\n",
       "      <td>divorced</td>\n",
       "      <td>professional.course</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>cellular</td>\n",
       "      <td>aug</td>\n",
       "      <td>mon</td>\n",
       "      <td>4621</td>\n",
       "      <td>1</td>\n",
       "      <td>112</td>\n",
       "      <td>2</td>\n",
       "      <td>failure</td>\n",
       "      <td>1.4</td>\n",
       "      <td>90.81</td>\n",
       "      <td>-35.53</td>\n",
       "      <td>0.69</td>\n",
       "      <td>5219.74</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>50</td>\n",
       "      <td>services</td>\n",
       "      <td>married</td>\n",
       "      <td>high.school</td>\n",
       "      <td>unknown</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>may</td>\n",
       "      <td>mon</td>\n",
       "      <td>4715</td>\n",
       "      <td>1</td>\n",
       "      <td>412</td>\n",
       "      <td>2</td>\n",
       "      <td>nonexistent</td>\n",
       "      <td>-1.8</td>\n",
       "      <td>96.33</td>\n",
       "      <td>-40.58</td>\n",
       "      <td>4.05</td>\n",
       "      <td>4974.79</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>48</td>\n",
       "      <td>blue-collar</td>\n",
       "      <td>divorced</td>\n",
       "      <td>basic.9y</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>apr</td>\n",
       "      <td>wed</td>\n",
       "      <td>171</td>\n",
       "      <td>0</td>\n",
       "      <td>1027</td>\n",
       "      <td>1</td>\n",
       "      <td>failure</td>\n",
       "      <td>-1.8</td>\n",
       "      <td>96.33</td>\n",
       "      <td>-44.74</td>\n",
       "      <td>1.50</td>\n",
       "      <td>5022.61</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>26</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>single</td>\n",
       "      <td>high.school</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>yes</td>\n",
       "      <td>cellular</td>\n",
       "      <td>aug</td>\n",
       "      <td>fri</td>\n",
       "      <td>359</td>\n",
       "      <td>26</td>\n",
       "      <td>998</td>\n",
       "      <td>0</td>\n",
       "      <td>nonexistent</td>\n",
       "      <td>1.4</td>\n",
       "      <td>97.08</td>\n",
       "      <td>-35.55</td>\n",
       "      <td>5.11</td>\n",
       "      <td>5222.87</td>\n",
       "      <td>yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>45</td>\n",
       "      <td>admin.</td>\n",
       "      <td>single</td>\n",
       "      <td>university.degree</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>cellular</td>\n",
       "      <td>nov</td>\n",
       "      <td>tue</td>\n",
       "      <td>3178</td>\n",
       "      <td>1</td>\n",
       "      <td>240</td>\n",
       "      <td>4</td>\n",
       "      <td>success</td>\n",
       "      <td>-3.4</td>\n",
       "      <td>89.82</td>\n",
       "      <td>-33.83</td>\n",
       "      <td>1.17</td>\n",
       "      <td>4884.70</td>\n",
       "      <td>no</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  age           job   marital            education  default housing loan   contact month day_of_week  duration  campaign  pdays  previous     poutcome  emp_var_rate  cons_price_index  cons_conf_index  lending_rate3m  nr_employed subscribe\n",
       "0   1   51        admin.  divorced  professional.course       no     yes  yes  cellular   aug         mon      4621         1    112         2      failure           1.4             90.81           -35.53            0.69      5219.74        no\n",
       "1   2   50      services   married          high.school  unknown     yes   no  cellular   may         mon      4715         1    412         2  nonexistent          -1.8             96.33           -40.58            4.05      4974.79       yes\n",
       "2   3   48   blue-collar  divorced             basic.9y       no      no   no  cellular   apr         wed       171         0   1027         1      failure          -1.8             96.33           -44.74            1.50      5022.61        no\n",
       "3   4   26  entrepreneur    single          high.school      yes     yes  yes  cellular   aug         fri       359        26    998         0  nonexistent           1.4             97.08           -35.55            5.11      5222.87       yes\n",
       "4   5   45        admin.    single    university.degree       no      no   no  cellular   nov         tue      3178         1    240         4      success          -3.4             89.82           -33.83            1.17      4884.70        no"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(\"../dataset/train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0afaa164",
   "metadata": {},
   "source": [
    "# 特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ca487210",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-18T11:34:40.797048Z",
     "start_time": "2023-11-18T11:34:40.670514Z"
    }
   },
   "outputs": [],
   "source": [
    "# 列出需要标准化的数值型特征和需要独热编码的类别型特征\n",
    "numeric_features = ['age', 'duration', 'campaign', 'pdays', 'previous', 'emp_var_rate', \n",
    "                    'cons_price_index', 'cons_conf_index', 'lending_rate3m', 'nr_employed']\n",
    "categorical_features = ['job', 'marital', 'education', 'default', 'housing', 'loan', 'contact', 'month', 'day_of_week', 'poutcome']\n",
    "# 定义ColumnTransformer\n",
    "preprocessor = ColumnTransformer(\n",
    "    transformers=[\n",
    "        ('poly', PolynomialFeatures(degree=3, include_bias=False), numeric_features),  # 标准化数值型特征\n",
    "        ('cat', OneHotEncoder(), categorical_features)  # 独热编码类别型特征\n",
    "    ])\n",
    "\n",
    "# 定义包含数据处理和特征工程的Pipeline\n",
    "preprocessing_pipeline = Pipeline([\n",
    "    ('preprocessor', preprocessor)  # 数据处理和特征工程\n",
    "])\n",
    "\n",
    "# 在训练集上拟合数据处理和特征工程的Pipeline\n",
    "X_transformed = preprocessing_pipeline.fit_transform(train)\n",
    "\n",
    "y = train[\"subscribe\"].apply(lambda x: 1 if x == \"yes\" else 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f336a187",
   "metadata": {},
   "source": [
    "# 贝叶斯搜索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "853c2e98",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-18T11:41:08.614949Z",
     "start_time": "2023-11-18T11:35:05.966884Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters found:  OrderedDict([('class_weight_ratio', 6), ('colsample_bytree', 0.6308093864100461), ('importance_type', 'split'), ('learning_rate', 0.022858304724422775), ('max_depth', 14), ('min_child_samples', 20), ('min_child_weight', 1.2657660576147527), ('min_split_gain', 1.0), ('n_estimators', 132), ('num_leaves', 35), ('reg_alpha', 0.34232009882038966), ('reg_lambda', 0.43458471477802046), ('subsample', 0.8907117965787229), ('subsample_for_bin', 168097), ('subsample_freq', 6)])\n",
      "Best cross-validation score: -0.8909\n",
      "CPU times: total: 4min 8s\n",
      "Wall time: 6min 2s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# 定义参数搜索范围\n",
    "param_space = {\n",
    "    'num_leaves': (5, 50),\n",
    "    'max_depth': (3, 15),\n",
    "    'learning_rate': (0.01, 0.2, 'log-uniform'),\n",
    "    'n_estimators': (50, 200),\n",
    "    'subsample_for_bin': (20000, 300000),\n",
    "    'class_weight_ratio': (1, 10),\n",
    "    'min_split_gain': (0.0, 1.0),\n",
    "    'min_child_weight': (0.5, 10),\n",
    "    'min_child_samples': (5, 20),\n",
    "    'subsample': (0.5, 1.0),\n",
    "    'subsample_freq': (1, 10),\n",
    "    'colsample_bytree': (0.5, 1.0),\n",
    "    'reg_alpha': (0.0, 1.0),\n",
    "    'reg_lambda': (0.0, 1.0),\n",
    "    'importance_type': ['split', 'gain']\n",
    "}\n",
    "\n",
    "lgbm_model = LGBMClassifier(boosting_type=\"gbdt\", objective=\"binary\", random_state=12)\n",
    "\n",
    "def objective_function(params):\n",
    "    # 将权重比例转换为字典形式\n",
    "    class_weight = {0: 1, 1: params['class_weight_ratio']}\n",
    "    params.pop('class_weight_ratio')  # 从参数中移除权重比例\n",
    "    lgbm_model.set_params(class_weight=class_weight, **params)\n",
    "    return -np.mean(cross_val_score(lgbm_model, X_transformed, y, cv=5, scoring='roc_auc'))\n",
    "\n",
    "# 使用BayesSearchCV进行贝叶斯调参\n",
    "bayes_search = BayesSearchCV(\n",
    "    lgbm_model,\n",
    "    param_space,\n",
    "    n_iter=50,\n",
    "    cv=5,\n",
    "    scoring='roc_auc',\n",
    "    random_state=42,\n",
    "    n_jobs=-1\n",
    ")\n",
    "\n",
    "# 执行贝叶斯搜索\n",
    "bayes_search.fit(X_transformed, y)\n",
    "\n",
    "# 输出最佳参数\n",
    "print(\"Best parameters found: \", bayes_search.best_params_)\n",
    "\n",
    "# 输出最佳交叉验证分数\n",
    "print(\"Best cross-validation score: {:.4f}\".format(-bayes_search.best_score_))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "daf442a2",
   "metadata": {},
   "source": [
    "# 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "4bdd6957",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-18T11:43:00.610532Z",
     "start_time": "2023-11-18T11:42:59.292327Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1]\ttraining's binary_logloss: 0.38306\ttraining's auc: 0.870385\tvalid_1's binary_logloss: 0.379432\tvalid_1's auc: 0.857163\n",
      "[2]\ttraining's binary_logloss: 0.376582\ttraining's auc: 0.879533\tvalid_1's binary_logloss: 0.373654\tvalid_1's auc: 0.863683\n",
      "[3]\ttraining's binary_logloss: 0.370562\ttraining's auc: 0.890197\tvalid_1's binary_logloss: 0.36828\tvalid_1's auc: 0.871294\n",
      "[4]\ttraining's binary_logloss: 0.364961\ttraining's auc: 0.893263\tvalid_1's binary_logloss: 0.363251\tvalid_1's auc: 0.872811\n",
      "[5]\ttraining's binary_logloss: 0.359834\ttraining's auc: 0.893181\tvalid_1's binary_logloss: 0.358787\tvalid_1's auc: 0.87377\n",
      "[6]\ttraining's binary_logloss: 0.354913\ttraining's auc: 0.895521\tvalid_1's binary_logloss: 0.354325\tvalid_1's auc: 0.874383\n",
      "[7]\ttraining's binary_logloss: 0.350353\ttraining's auc: 0.897814\tvalid_1's binary_logloss: 0.350287\tvalid_1's auc: 0.875273\n",
      "[8]\ttraining's binary_logloss: 0.346114\ttraining's auc: 0.898519\tvalid_1's binary_logloss: 0.346571\tvalid_1's auc: 0.875492\n",
      "[9]\ttraining's binary_logloss: 0.341909\ttraining's auc: 0.899792\tvalid_1's binary_logloss: 0.342973\tvalid_1's auc: 0.876698\n",
      "[10]\ttraining's binary_logloss: 0.338027\ttraining's auc: 0.900514\tvalid_1's binary_logloss: 0.33963\tvalid_1's auc: 0.87786\n",
      "[11]\ttraining's binary_logloss: 0.334283\ttraining's auc: 0.901186\tvalid_1's binary_logloss: 0.336379\tvalid_1's auc: 0.878029\n",
      "[12]\ttraining's binary_logloss: 0.330818\ttraining's auc: 0.901764\tvalid_1's binary_logloss: 0.33344\tvalid_1's auc: 0.878551\n",
      "[13]\ttraining's binary_logloss: 0.327486\ttraining's auc: 0.903019\tvalid_1's binary_logloss: 0.330525\tvalid_1's auc: 0.87877\n",
      "[14]\ttraining's binary_logloss: 0.324297\ttraining's auc: 0.903438\tvalid_1's binary_logloss: 0.327767\tvalid_1's auc: 0.87885\n",
      "[15]\ttraining's binary_logloss: 0.321279\ttraining's auc: 0.903715\tvalid_1's binary_logloss: 0.325237\tvalid_1's auc: 0.878785\n",
      "[16]\ttraining's binary_logloss: 0.318348\ttraining's auc: 0.90416\tvalid_1's binary_logloss: 0.322744\tvalid_1's auc: 0.878545\n",
      "[17]\ttraining's binary_logloss: 0.315564\ttraining's auc: 0.904453\tvalid_1's binary_logloss: 0.320428\tvalid_1's auc: 0.878144\n",
      "[18]\ttraining's binary_logloss: 0.312922\ttraining's auc: 0.904544\tvalid_1's binary_logloss: 0.318155\tvalid_1's auc: 0.878345\n",
      "[19]\ttraining's binary_logloss: 0.310338\ttraining's auc: 0.905728\tvalid_1's binary_logloss: 0.316042\tvalid_1's auc: 0.879219\n",
      "[20]\ttraining's binary_logloss: 0.30773\ttraining's auc: 0.906761\tvalid_1's binary_logloss: 0.314045\tvalid_1's auc: 0.878933\n",
      "[21]\ttraining's binary_logloss: 0.305303\ttraining's auc: 0.907237\tvalid_1's binary_logloss: 0.312064\tvalid_1's auc: 0.879445\n",
      "[22]\ttraining's binary_logloss: 0.302927\ttraining's auc: 0.907673\tvalid_1's binary_logloss: 0.310167\tvalid_1's auc: 0.879544\n",
      "[23]\ttraining's binary_logloss: 0.300637\ttraining's auc: 0.907977\tvalid_1's binary_logloss: 0.30828\tvalid_1's auc: 0.879797\n",
      "[24]\ttraining's binary_logloss: 0.298462\ttraining's auc: 0.908237\tvalid_1's binary_logloss: 0.30647\tvalid_1's auc: 0.880103\n",
      "[25]\ttraining's binary_logloss: 0.296342\ttraining's auc: 0.909084\tvalid_1's binary_logloss: 0.304792\tvalid_1's auc: 0.880191\n",
      "[26]\ttraining's binary_logloss: 0.294349\ttraining's auc: 0.910224\tvalid_1's binary_logloss: 0.303217\tvalid_1's auc: 0.881047\n",
      "[27]\ttraining's binary_logloss: 0.292393\ttraining's auc: 0.910409\tvalid_1's binary_logloss: 0.301691\tvalid_1's auc: 0.881178\n",
      "[28]\ttraining's binary_logloss: 0.29054\ttraining's auc: 0.910874\tvalid_1's binary_logloss: 0.300325\tvalid_1's auc: 0.881188\n",
      "[29]\ttraining's binary_logloss: 0.288759\ttraining's auc: 0.912092\tvalid_1's binary_logloss: 0.299026\tvalid_1's auc: 0.881212\n",
      "[30]\ttraining's binary_logloss: 0.286981\ttraining's auc: 0.912539\tvalid_1's binary_logloss: 0.297665\tvalid_1's auc: 0.881368\n",
      "[31]\ttraining's binary_logloss: 0.285208\ttraining's auc: 0.912902\tvalid_1's binary_logloss: 0.296292\tvalid_1's auc: 0.881608\n",
      "[32]\ttraining's binary_logloss: 0.283512\ttraining's auc: 0.913173\tvalid_1's binary_logloss: 0.294965\tvalid_1's auc: 0.882219\n",
      "[33]\ttraining's binary_logloss: 0.281854\ttraining's auc: 0.913818\tvalid_1's binary_logloss: 0.29376\tvalid_1's auc: 0.882534\n",
      "[34]\ttraining's binary_logloss: 0.280269\ttraining's auc: 0.91405\tvalid_1's binary_logloss: 0.292584\tvalid_1's auc: 0.882553\n",
      "[35]\ttraining's binary_logloss: 0.278672\ttraining's auc: 0.914423\tvalid_1's binary_logloss: 0.291361\tvalid_1's auc: 0.882866\n",
      "[36]\ttraining's binary_logloss: 0.27719\ttraining's auc: 0.914739\tvalid_1's binary_logloss: 0.290321\tvalid_1's auc: 0.88295\n",
      "[37]\ttraining's binary_logloss: 0.275777\ttraining's auc: 0.915445\tvalid_1's binary_logloss: 0.289372\tvalid_1's auc: 0.88287\n",
      "[38]\ttraining's binary_logloss: 0.27433\ttraining's auc: 0.91593\tvalid_1's binary_logloss: 0.288355\tvalid_1's auc: 0.882923\n",
      "[39]\ttraining's binary_logloss: 0.272968\ttraining's auc: 0.916217\tvalid_1's binary_logloss: 0.287403\tvalid_1's auc: 0.882954\n",
      "[40]\ttraining's binary_logloss: 0.271634\ttraining's auc: 0.916675\tvalid_1's binary_logloss: 0.286543\tvalid_1's auc: 0.882987\n",
      "[41]\ttraining's binary_logloss: 0.270339\ttraining's auc: 0.917229\tvalid_1's binary_logloss: 0.2856\tvalid_1's auc: 0.88316\n",
      "[42]\ttraining's binary_logloss: 0.269064\ttraining's auc: 0.917629\tvalid_1's binary_logloss: 0.284687\tvalid_1's auc: 0.883225\n",
      "[43]\ttraining's binary_logloss: 0.267844\ttraining's auc: 0.918114\tvalid_1's binary_logloss: 0.283809\tvalid_1's auc: 0.883267\n",
      "[44]\ttraining's binary_logloss: 0.266554\ttraining's auc: 0.918496\tvalid_1's binary_logloss: 0.282939\tvalid_1's auc: 0.883303\n",
      "[45]\ttraining's binary_logloss: 0.265294\ttraining's auc: 0.919113\tvalid_1's binary_logloss: 0.28205\tvalid_1's auc: 0.883695\n",
      "[46]\ttraining's binary_logloss: 0.264158\ttraining's auc: 0.919565\tvalid_1's binary_logloss: 0.281358\tvalid_1's auc: 0.883534\n",
      "[47]\ttraining's binary_logloss: 0.26306\ttraining's auc: 0.919867\tvalid_1's binary_logloss: 0.280607\tvalid_1's auc: 0.883601\n",
      "[48]\ttraining's binary_logloss: 0.261932\ttraining's auc: 0.920461\tvalid_1's binary_logloss: 0.279833\tvalid_1's auc: 0.883868\n",
      "[49]\ttraining's binary_logloss: 0.260856\ttraining's auc: 0.920714\tvalid_1's binary_logloss: 0.279145\tvalid_1's auc: 0.883856\n",
      "[50]\ttraining's binary_logloss: 0.25978\ttraining's auc: 0.920984\tvalid_1's binary_logloss: 0.278438\tvalid_1's auc: 0.883958\n",
      "[51]\ttraining's binary_logloss: 0.258781\ttraining's auc: 0.921274\tvalid_1's binary_logloss: 0.277791\tvalid_1's auc: 0.883932\n",
      "[52]\ttraining's binary_logloss: 0.257807\ttraining's auc: 0.921393\tvalid_1's binary_logloss: 0.277216\tvalid_1's auc: 0.88384\n",
      "[53]\ttraining's binary_logloss: 0.256796\ttraining's auc: 0.921782\tvalid_1's binary_logloss: 0.276559\tvalid_1's auc: 0.884049\n",
      "[54]\ttraining's binary_logloss: 0.25581\ttraining's auc: 0.922065\tvalid_1's binary_logloss: 0.276035\tvalid_1's auc: 0.883966\n",
      "[55]\ttraining's binary_logloss: 0.254878\ttraining's auc: 0.922483\tvalid_1's binary_logloss: 0.275526\tvalid_1's auc: 0.883805\n",
      "[56]\ttraining's binary_logloss: 0.253986\ttraining's auc: 0.922713\tvalid_1's binary_logloss: 0.27497\tvalid_1's auc: 0.883839\n",
      "[57]\ttraining's binary_logloss: 0.253113\ttraining's auc: 0.923022\tvalid_1's binary_logloss: 0.274382\tvalid_1's auc: 0.883933\n",
      "[58]\ttraining's binary_logloss: 0.252226\ttraining's auc: 0.923216\tvalid_1's binary_logloss: 0.273859\tvalid_1's auc: 0.884029\n",
      "[59]\ttraining's binary_logloss: 0.25137\ttraining's auc: 0.923355\tvalid_1's binary_logloss: 0.273327\tvalid_1's auc: 0.884158\n",
      "[60]\ttraining's binary_logloss: 0.250475\ttraining's auc: 0.923676\tvalid_1's binary_logloss: 0.272833\tvalid_1's auc: 0.884148\n",
      "[61]\ttraining's binary_logloss: 0.249619\ttraining's auc: 0.92406\tvalid_1's binary_logloss: 0.272359\tvalid_1's auc: 0.884356\n",
      "[62]\ttraining's binary_logloss: 0.248778\ttraining's auc: 0.924346\tvalid_1's binary_logloss: 0.271865\tvalid_1's auc: 0.884474\n",
      "[63]\ttraining's binary_logloss: 0.247917\ttraining's auc: 0.924667\tvalid_1's binary_logloss: 0.271419\tvalid_1's auc: 0.884447\n",
      "[64]\ttraining's binary_logloss: 0.247099\ttraining's auc: 0.924926\tvalid_1's binary_logloss: 0.270951\tvalid_1's auc: 0.884513\n",
      "[65]\ttraining's binary_logloss: 0.246286\ttraining's auc: 0.925302\tvalid_1's binary_logloss: 0.270506\tvalid_1's auc: 0.884528\n",
      "[66]\ttraining's binary_logloss: 0.245496\ttraining's auc: 0.925639\tvalid_1's binary_logloss: 0.270108\tvalid_1's auc: 0.884495\n",
      "[67]\ttraining's binary_logloss: 0.244737\ttraining's auc: 0.925959\tvalid_1's binary_logloss: 0.269713\tvalid_1's auc: 0.884521\n",
      "[68]\ttraining's binary_logloss: 0.24398\ttraining's auc: 0.926328\tvalid_1's binary_logloss: 0.269318\tvalid_1's auc: 0.884597\n",
      "[69]\ttraining's binary_logloss: 0.243224\ttraining's auc: 0.926708\tvalid_1's binary_logloss: 0.268973\tvalid_1's auc: 0.88464\n",
      "[70]\ttraining's binary_logloss: 0.242481\ttraining's auc: 0.927146\tvalid_1's binary_logloss: 0.268667\tvalid_1's auc: 0.884679\n",
      "[71]\ttraining's binary_logloss: 0.241772\ttraining's auc: 0.927369\tvalid_1's binary_logloss: 0.268309\tvalid_1's auc: 0.884652\n",
      "[72]\ttraining's binary_logloss: 0.241084\ttraining's auc: 0.927586\tvalid_1's binary_logloss: 0.267944\tvalid_1's auc: 0.88475\n",
      "[73]\ttraining's binary_logloss: 0.240403\ttraining's auc: 0.927867\tvalid_1's binary_logloss: 0.267596\tvalid_1's auc: 0.884825\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[74]\ttraining's binary_logloss: 0.239733\ttraining's auc: 0.92816\tvalid_1's binary_logloss: 0.267279\tvalid_1's auc: 0.884824\n",
      "[75]\ttraining's binary_logloss: 0.239053\ttraining's auc: 0.928607\tvalid_1's binary_logloss: 0.267017\tvalid_1's auc: 0.884931\n",
      "[76]\ttraining's binary_logloss: 0.238392\ttraining's auc: 0.928939\tvalid_1's binary_logloss: 0.26665\tvalid_1's auc: 0.885129\n",
      "[77]\ttraining's binary_logloss: 0.237756\ttraining's auc: 0.929233\tvalid_1's binary_logloss: 0.266293\tvalid_1's auc: 0.885225\n",
      "[78]\ttraining's binary_logloss: 0.23712\ttraining's auc: 0.92959\tvalid_1's binary_logloss: 0.265988\tvalid_1's auc: 0.885394\n",
      "[79]\ttraining's binary_logloss: 0.236498\ttraining's auc: 0.929838\tvalid_1's binary_logloss: 0.265771\tvalid_1's auc: 0.885361\n",
      "[80]\ttraining's binary_logloss: 0.235874\ttraining's auc: 0.930087\tvalid_1's binary_logloss: 0.265477\tvalid_1's auc: 0.885415\n",
      "[81]\ttraining's binary_logloss: 0.235298\ttraining's auc: 0.930278\tvalid_1's binary_logloss: 0.26512\tvalid_1's auc: 0.885565\n",
      "[82]\ttraining's binary_logloss: 0.234694\ttraining's auc: 0.930575\tvalid_1's binary_logloss: 0.264809\tvalid_1's auc: 0.885689\n",
      "[83]\ttraining's binary_logloss: 0.234133\ttraining's auc: 0.930783\tvalid_1's binary_logloss: 0.264482\tvalid_1's auc: 0.885858\n",
      "[84]\ttraining's binary_logloss: 0.233554\ttraining's auc: 0.931015\tvalid_1's binary_logloss: 0.264241\tvalid_1's auc: 0.885918\n",
      "[85]\ttraining's binary_logloss: 0.232985\ttraining's auc: 0.93128\tvalid_1's binary_logloss: 0.26402\tvalid_1's auc: 0.885959\n",
      "[86]\ttraining's binary_logloss: 0.232403\ttraining's auc: 0.931549\tvalid_1's binary_logloss: 0.263826\tvalid_1's auc: 0.885979\n",
      "[87]\ttraining's binary_logloss: 0.231856\ttraining's auc: 0.931807\tvalid_1's binary_logloss: 0.263595\tvalid_1's auc: 0.886063\n",
      "[88]\ttraining's binary_logloss: 0.231318\ttraining's auc: 0.93214\tvalid_1's binary_logloss: 0.263414\tvalid_1's auc: 0.886082\n",
      "[89]\ttraining's binary_logloss: 0.23076\ttraining's auc: 0.932434\tvalid_1's binary_logloss: 0.263202\tvalid_1's auc: 0.88619\n",
      "[90]\ttraining's binary_logloss: 0.23022\ttraining's auc: 0.932674\tvalid_1's binary_logloss: 0.262988\tvalid_1's auc: 0.886225\n",
      "[91]\ttraining's binary_logloss: 0.229664\ttraining's auc: 0.932945\tvalid_1's binary_logloss: 0.262797\tvalid_1's auc: 0.886234\n",
      "[92]\ttraining's binary_logloss: 0.229118\ttraining's auc: 0.933231\tvalid_1's binary_logloss: 0.262642\tvalid_1's auc: 0.886178\n",
      "[93]\ttraining's binary_logloss: 0.2286\ttraining's auc: 0.933442\tvalid_1's binary_logloss: 0.262463\tvalid_1's auc: 0.886202\n",
      "[94]\ttraining's binary_logloss: 0.228071\ttraining's auc: 0.93373\tvalid_1's binary_logloss: 0.262285\tvalid_1's auc: 0.886266\n",
      "[95]\ttraining's binary_logloss: 0.227569\ttraining's auc: 0.933979\tvalid_1's binary_logloss: 0.262151\tvalid_1's auc: 0.886222\n",
      "[96]\ttraining's binary_logloss: 0.227055\ttraining's auc: 0.934248\tvalid_1's binary_logloss: 0.26198\tvalid_1's auc: 0.886319\n",
      "[97]\ttraining's binary_logloss: 0.226579\ttraining's auc: 0.934611\tvalid_1's binary_logloss: 0.261825\tvalid_1's auc: 0.886364\n",
      "[98]\ttraining's binary_logloss: 0.226055\ttraining's auc: 0.934995\tvalid_1's binary_logloss: 0.261694\tvalid_1's auc: 0.886383\n",
      "[99]\ttraining's binary_logloss: 0.225557\ttraining's auc: 0.935266\tvalid_1's binary_logloss: 0.2615\tvalid_1's auc: 0.886526\n",
      "[100]\ttraining's binary_logloss: 0.225092\ttraining's auc: 0.93563\tvalid_1's binary_logloss: 0.261361\tvalid_1's auc: 0.886514\n",
      "[101]\ttraining's binary_logloss: 0.224617\ttraining's auc: 0.935921\tvalid_1's binary_logloss: 0.261252\tvalid_1's auc: 0.886461\n",
      "[102]\ttraining's binary_logloss: 0.2241\ttraining's auc: 0.936234\tvalid_1's binary_logloss: 0.261141\tvalid_1's auc: 0.886437\n",
      "[103]\ttraining's binary_logloss: 0.22366\ttraining's auc: 0.936483\tvalid_1's binary_logloss: 0.261049\tvalid_1's auc: 0.886403\n",
      "[104]\ttraining's binary_logloss: 0.223235\ttraining's auc: 0.936668\tvalid_1's binary_logloss: 0.260929\tvalid_1's auc: 0.886336\n",
      "[105]\ttraining's binary_logloss: 0.22274\ttraining's auc: 0.936957\tvalid_1's binary_logloss: 0.260865\tvalid_1's auc: 0.886312\n",
      "[106]\ttraining's binary_logloss: 0.222298\ttraining's auc: 0.937159\tvalid_1's binary_logloss: 0.260727\tvalid_1's auc: 0.886399\n",
      "[107]\ttraining's binary_logloss: 0.221849\ttraining's auc: 0.937412\tvalid_1's binary_logloss: 0.260569\tvalid_1's auc: 0.886545\n",
      "[108]\ttraining's binary_logloss: 0.221367\ttraining's auc: 0.93769\tvalid_1's binary_logloss: 0.260416\tvalid_1's auc: 0.886647\n",
      "[109]\ttraining's binary_logloss: 0.220897\ttraining's auc: 0.937964\tvalid_1's binary_logloss: 0.260236\tvalid_1's auc: 0.886707\n",
      "[110]\ttraining's binary_logloss: 0.220386\ttraining's auc: 0.93829\tvalid_1's binary_logloss: 0.260101\tvalid_1's auc: 0.886753\n",
      "[111]\ttraining's binary_logloss: 0.219952\ttraining's auc: 0.938521\tvalid_1's binary_logloss: 0.26004\tvalid_1's auc: 0.886722\n",
      "[112]\ttraining's binary_logloss: 0.219543\ttraining's auc: 0.938739\tvalid_1's binary_logloss: 0.259964\tvalid_1's auc: 0.886713\n",
      "[113]\ttraining's binary_logloss: 0.219051\ttraining's auc: 0.939018\tvalid_1's binary_logloss: 0.259791\tvalid_1's auc: 0.886796\n",
      "[114]\ttraining's binary_logloss: 0.21864\ttraining's auc: 0.939265\tvalid_1's binary_logloss: 0.259709\tvalid_1's auc: 0.886822\n",
      "[115]\ttraining's binary_logloss: 0.218174\ttraining's auc: 0.939552\tvalid_1's binary_logloss: 0.259647\tvalid_1's auc: 0.886776\n",
      "[116]\ttraining's binary_logloss: 0.217739\ttraining's auc: 0.939801\tvalid_1's binary_logloss: 0.25955\tvalid_1's auc: 0.886755\n",
      "[117]\ttraining's binary_logloss: 0.217271\ttraining's auc: 0.940098\tvalid_1's binary_logloss: 0.259416\tvalid_1's auc: 0.886797\n",
      "[118]\ttraining's binary_logloss: 0.216826\ttraining's auc: 0.940378\tvalid_1's binary_logloss: 0.259389\tvalid_1's auc: 0.886719\n",
      "[119]\ttraining's binary_logloss: 0.216389\ttraining's auc: 0.940637\tvalid_1's binary_logloss: 0.259325\tvalid_1's auc: 0.886672\n",
      "[120]\ttraining's binary_logloss: 0.215975\ttraining's auc: 0.940912\tvalid_1's binary_logloss: 0.259208\tvalid_1's auc: 0.886752\n",
      "[121]\ttraining's binary_logloss: 0.2155\ttraining's auc: 0.941206\tvalid_1's binary_logloss: 0.259131\tvalid_1's auc: 0.886787\n",
      "[122]\ttraining's binary_logloss: 0.215066\ttraining's auc: 0.941449\tvalid_1's binary_logloss: 0.259123\tvalid_1's auc: 0.886695\n",
      "[123]\ttraining's binary_logloss: 0.214644\ttraining's auc: 0.941747\tvalid_1's binary_logloss: 0.259054\tvalid_1's auc: 0.886786\n",
      "[124]\ttraining's binary_logloss: 0.214228\ttraining's auc: 0.941975\tvalid_1's binary_logloss: 0.259017\tvalid_1's auc: 0.886797\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-7 {color: black;background-color: white;}#sk-container-id-7 pre{padding: 0;}#sk-container-id-7 div.sk-toggleable {background-color: white;}#sk-container-id-7 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-7 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-7 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-7 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-7 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-7 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-7 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-7 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-7 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-7 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-7 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-7 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-7 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-7 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-7 div.sk-item {position: relative;z-index: 1;}#sk-container-id-7 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-7 div.sk-item::before, #sk-container-id-7 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-7 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-7 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-7 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-7 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-7 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-7 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-7 div.sk-label-container {text-align: center;}#sk-container-id-7 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-7 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-7\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LGBMClassifier(class_weight_ratio=6, colsample_bytree=0.6308093864100461,\n",
       "               learning_rate=0.022858304724422775, max_depth=14,\n",
       "               min_child_weight=1.2657660576147527, min_split_gain=1.0,\n",
       "               n_estimators=132, num_leaves=35, objective=&#x27;binary&#x27;,\n",
       "               random_state=12, reg_alpha=0.34232009882038966,\n",
       "               reg_lambda=0.43458471477802046, subsample=0.8907117965787229,\n",
       "               subsample_for_bin=168097, subsample_freq=6)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" checked><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LGBMClassifier</label><div class=\"sk-toggleable__content\"><pre>LGBMClassifier(class_weight_ratio=6, colsample_bytree=0.6308093864100461,\n",
       "               learning_rate=0.022858304724422775, max_depth=14,\n",
       "               min_child_weight=1.2657660576147527, min_split_gain=1.0,\n",
       "               n_estimators=132, num_leaves=35, objective=&#x27;binary&#x27;,\n",
       "               random_state=12, reg_alpha=0.34232009882038966,\n",
       "               reg_lambda=0.43458471477802046, subsample=0.8907117965787229,\n",
       "               subsample_for_bin=168097, subsample_freq=6)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LGBMClassifier(class_weight_ratio=6, colsample_bytree=0.6308093864100461,\n",
       "               learning_rate=0.022858304724422775, max_depth=14,\n",
       "               min_child_weight=1.2657660576147527, min_split_gain=1.0,\n",
       "               n_estimators=132, num_leaves=35, objective='binary',\n",
       "               random_state=12, reg_alpha=0.34232009882038966,\n",
       "               reg_lambda=0.43458471477802046, subsample=0.8907117965787229,\n",
       "               subsample_for_bin=168097, subsample_freq=6)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 划分训练集与验证集\n",
    "x_train, x_valid, y_train, y_valid = train_test_split(X_transformed, y, test_size=0.3, random_state=12)\n",
    "\n",
    "# 得到贝叶斯搜索得到的最佳参数\n",
    "best_params = bayes_search.best_params_\n",
    "\n",
    "# 将最佳参数设置到LightGBM模型中\n",
    "lgbm_model.set_params(**best_params)\n",
    "\n",
    "# 训练模型\n",
    "lgbm_model.fit(x_train, y_train, \n",
    "               eval_set=[(x_train, y_train), (x_valid, y_valid)], \n",
    "               eval_metric=['binary_logloss', 'auc'], \n",
    "               early_stopping_rounds=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4f58ffe",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-18T11:26:58.102173Z",
     "start_time": "2023-11-18T11:26:58.086017Z"
    }
   },
   "source": [
    "# 使用模型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c66d1743",
   "metadata": {},
   "source": [
    "模型得分：686/0.96"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "84bf41e2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-11-18T11:43:05.291430Z",
     "start_time": "2023-11-18T11:43:05.209583Z"
    }
   },
   "outputs": [],
   "source": [
    "test = pd.read_csv(\"../dataset/test.csv\")\n",
    "use_test = test.iloc[:, 1:]\n",
    "\n",
    "test_transformed = preprocessing_pipeline.transform(use_test)\n",
    "test[\"subscribe\"] = lgbm_model.predict(test_transformed)\n",
    "test[\"subscribe\"] = test[\"subscribe\"].apply(lambda x: \"yes\" if x == 1 else \"no\")\n",
    "\n",
    "test[[\"id\", \"subscribe\"]].to_csv(\"../results/submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07db7f09",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ML",
   "language": "python",
   "name": "ml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "264.875px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
